import re
import random
import requests
import readua


uas = readua.read_ua('UA.txt')

def get_html(url):

    headers = {
        'User-Agent': random.choice(uas)
    }
    # print(headers)
    try:
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        return r.text
    except:
        return ''


def get_books(surl):
    page = get_html(surl)
    pattern = re.compile(r'<a href="(/book/\d+/)">(\w+)</a>')
    bs = pattern.findall(page)
    books = []
    for item in bs:
        c = surl + item[0]
        books.append({
            'link': c,
            'title': item[1]
        })
   
    return books


def get_chapters(burl):
    page = get_html(burl)
    pattern = re.compile(r'<dd><a href="/book/\d+/(\d+\.html)"  >(第.*?)</a></dd>')
    cs = pattern.findall(page)
    chapters = []
    for item in cs:
        c = burl + item[0]
        chapters.append({
            'link': c,
            'title': item[1]
        })
    # print(chapters)
    return chapters


def get_content(curl):
    page = get_html(curl)
    pattern = re.compile(r'<div id="content">(.*?)</div>')
    content = pattern.findall(page)[0]
    return content


if __name__ == '__main__':
    # burl = 'https://www.81zw.com/book/10150/'
    surl = 'https://www.81zw.com'
    books = get_books(surl)
    print(books)

   

